{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a9bff53e",
   "metadata": {},
   "source": [
    "# Lesson 1: Advanced RAG Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "feb98470-c136-471d-a63e-d50d8eb09c57",
   "metadata": {
    "height": 98,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ In Answer Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n",
      "✅ In Answer Relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n",
      "✅ In Context Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n",
      "✅ In Context Relevance, input response will be set to __record__.app.query.rets.source_nodes[:].node.text .\n",
      "✅ In Groundedness, input source will be set to __record__.app.query.rets.source_nodes[:].node.text .\n",
      "✅ In Groundedness, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "\n",
    "import os\n",
    "import openai\n",
    "openai.api_key = utils.get_openai_api_key()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "31e2859b-596e-40b3-867b-f4d6e91f74bc",
   "metadata": {
    "height": 98,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index import SimpleDirectoryReader\n",
    "\n",
    "documents = SimpleDirectoryReader(\n",
    "    input_files=[\"./eBook-How-to-Build-a-Career-in-AI.pdf\"]\n",
    ").load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2d7d0857-b9d1-4feb-8243-bfd2f4953acd",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'> \n",
      "\n",
      "41 \n",
      "\n",
      "<class 'llama_index.schema.Document'>\n",
      "Doc ID: 56503002-cc6c-4a63-9127-9fd3399a0f09\n",
      "Text: PAGE 1Founder, DeepLearning.AICollected Insights from Andrew Ng\n",
      "How to  Build Your Career in AIA Simple Guide\n"
     ]
    }
   ],
   "source": [
    "print(type(documents), \"\\n\")\n",
    "print(len(documents), \"\\n\")\n",
    "print(type(documents[0]))\n",
    "print(documents[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3123d3d",
   "metadata": {},
   "source": [
    "## Basic RAG pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b4abc806-64f5-46bb-8c9f-6469ecb18d20",
   "metadata": {
    "height": 64,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index import Document\n",
    "\n",
    "document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "afc2baff-5e8b-4733-9899-16f248777b23",
   "metadata": {
    "height": 183,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d1106174deb4412b27af8575aeb6d27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/743 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94b9062572da4e3d8270005f52e79a08",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8eb095dcac5b448883d859180c88b548",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/366 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa649627461a49e282e8c9504858372e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "230fcc2aa4a04ebf90be812049e2477f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/711k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ce9c2935321435bb518712d476503cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /tmp/llama_index...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "from llama_index import VectorStoreIndex\n",
    "from llama_index import ServiceContext\n",
    "from llama_index.llms import OpenAI\n",
    "\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1)\n",
    "service_context = ServiceContext.from_defaults(\n",
    "    llm=llm, embed_model=\"local:BAAI/bge-small-en-v1.5\"\n",
    ")\n",
    "index = VectorStoreIndex.from_documents([document],\n",
    "                                        service_context=service_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae52a26c-7d0c-44df-8043-4c7f19f794b9",
   "metadata": {
    "height": 30,
    "tags": []
   },
   "outputs": [],
   "source": [
    "query_engine = index.as_query_engine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6b0d5b6e-cc2e-4648-b28c-5fa25a97d175",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Develop a side hustle, ensure the project will help you grow technically, collaborate with good teammates, and consider if the project can be a stepping stone to larger projects.\n"
     ]
    }
   ],
   "source": [
    "response = query_engine.query(\n",
    "    \"What are steps to take when finding projects to build your experience?\"\n",
    ")\n",
    "print(str(response))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e50a1ac5",
   "metadata": {},
   "source": [
    "## Evaluation setup using TruLens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0ead7dc1-71b2-4001-918f-bf8d610fd3fd",
   "metadata": {
    "height": 132,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are the keys to building a career in AI?\n",
      "How can teamwork contribute to success in AI?\n",
      "What is the importance of networking in AI?\n",
      "What are some good habits to develop for a successful career?\n",
      "How can altruism be beneficial in building a career?\n",
      "What is imposter syndrome and how does it relate to AI?\n",
      "Who are some accomplished individuals who have experienced imposter syndrome?\n",
      "What is the first step to becoming good at AI?\n",
      "What are some common challenges in AI?\n",
      "Is it normal to find parts of AI challenging?\n"
     ]
    }
   ],
   "source": [
    "eval_questions = []\n",
    "with open('eval_questions.txt', 'r') as file:\n",
    "    for line in file:\n",
    "        # Remove newline character and convert to integer\n",
    "        item = line.strip()\n",
    "        print(item)\n",
    "        eval_questions.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "87a278f8",
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "# You can try your own question:\n",
    "new_question = \"What is the right AI job for me?\"\n",
    "eval_questions.append(new_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8d5204e8",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['What are the keys to building a career in AI?', 'How can teamwork contribute to success in AI?', 'What is the importance of networking in AI?', 'What are some good habits to develop for a successful career?', 'How can altruism be beneficial in building a career?', 'What is imposter syndrome and how does it relate to AI?', 'Who are some accomplished individuals who have experienced imposter syndrome?', 'What is the first step to becoming good at AI?', 'What are some common challenges in AI?', 'Is it normal to find parts of AI challenging?', 'What is the right AI job for me?']\n"
     ]
    }
   ],
   "source": [
    "print(eval_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c063c9c7-bf1e-4b24-9a22-d4281c0f954e",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🦑 Tru initialized with db url sqlite:///default.sqlite .\n",
      "🛑 Secret keys may be written to the database. See the `database_redact_keys` option of `Tru` to prevent this.\n"
     ]
    }
   ],
   "source": [
    "from trulens_eval import Tru\n",
    "tru = Tru()\n",
    "\n",
    "tru.reset_database()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03802825-6ce4-4563-aeec-d8d57e095ad1",
   "metadata": {},
   "source": [
    "For the classroom, we've written some of the code in helper functions inside a utils.py file.  \n",
    "- You can view the utils.py file in the file directory by clicking on the \"Jupyter\" logo at the top of the notebook.\n",
    "- In later lessons, you'll get to work directly with the code that's currently wrapped inside these helper functions, to give you more options to customize your RAG pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4f754bed-d16f-4c8d-a1a1-b36096272570",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from utils import get_prebuilt_trulens_recorder\n",
    "\n",
    "tru_recorder = get_prebuilt_trulens_recorder(query_engine,\n",
    "                                             app_id=\"Direct Query Engine\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4dbdfbcc-aac7-4805-9894-4fc016c66bf6",
   "metadata": {
    "height": 64,
    "tags": []
   },
   "outputs": [],
   "source": [
    "with tru_recorder as recording:\n",
    "    for question in eval_questions:\n",
    "        response = query_engine.query(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e14f512b-601c-42d0-bfac-bf41d9c577e7",
   "metadata": {
    "height": 30,
    "tags": []
   },
   "outputs": [],
   "source": [
    "records, feedback = tru.get_records_and_feedback(app_ids=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2da4a602-0d56-4bf8-9fa6-03ef0b7e254b",
   "metadata": {
    "height": 30,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_id</th>\n",
       "      <th>app_json</th>\n",
       "      <th>type</th>\n",
       "      <th>record_id</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "      <th>tags</th>\n",
       "      <th>record_json</th>\n",
       "      <th>cost_json</th>\n",
       "      <th>perf_json</th>\n",
       "      <th>ts</th>\n",
       "      <th>Answer Relevance</th>\n",
       "      <th>Context Relevance</th>\n",
       "      <th>Groundedness</th>\n",
       "      <th>Answer Relevance_calls</th>\n",
       "      <th>Context Relevance_calls</th>\n",
       "      <th>Groundedness_calls</th>\n",
       "      <th>latency</th>\n",
       "      <th>total_tokens</th>\n",
       "      <th>total_cost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Direct Query Engine</td>\n",
       "      <td>{\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...</td>\n",
       "      <td>RetrieverQueryEngine(llama_index.query_engine....</td>\n",
       "      <td>record_hash_f7fff232a5841e153dd60e4e0e384fe5</td>\n",
       "      <td>\"What are the keys to building a career in AI?\"</td>\n",
       "      <td>\"Learning foundational technical skills, worki...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_f7fff232a5841e153dd...</td>\n",
       "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
       "      <td>{\"start_time\": \"2024-11-23T08:04:38.679417\", \"...</td>\n",
       "      <td>2024-11-23T08:04:39.735457</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>[{'args': {'prompt': 'What are the keys to bui...</td>\n",
       "      <td>[{'args': {'prompt': 'What are the keys to bui...</td>\n",
       "      <td>[{'args': {'source': 'PAGE 1Founder, DeepLearn...</td>\n",
       "      <td>1</td>\n",
       "      <td>2066</td>\n",
       "      <td>0.003123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Direct Query Engine</td>\n",
       "      <td>{\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...</td>\n",
       "      <td>RetrieverQueryEngine(llama_index.query_engine....</td>\n",
       "      <td>record_hash_c8c4775f87a3b231bbb60b4a2e2cc0b8</td>\n",
       "      <td>\"How can teamwork contribute to success in AI?\"</td>\n",
       "      <td>\"Teamwork can contribute to success in AI by a...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_c8c4775f87a3b231bbb...</td>\n",
       "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
       "      <td>{\"start_time\": \"2024-11-23T08:04:39.854287\", \"...</td>\n",
       "      <td>2024-11-23T08:04:40.930055</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>[{'args': {'prompt': 'How can teamwork contrib...</td>\n",
       "      <td>[{'args': {'prompt': 'How can teamwork contrib...</td>\n",
       "      <td>[{'args': {'source': 'Hopefully the previous c...</td>\n",
       "      <td>1</td>\n",
       "      <td>1676</td>\n",
       "      <td>0.002540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Direct Query Engine</td>\n",
       "      <td>{\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...</td>\n",
       "      <td>RetrieverQueryEngine(llama_index.query_engine....</td>\n",
       "      <td>record_hash_74173a7b0713c79a51d6ce6c4783522a</td>\n",
       "      <td>\"What is the importance of networking in AI?\"</td>\n",
       "      <td>\"Networking is crucial in AI as it helps indiv...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_74173a7b0713c79a51d...</td>\n",
       "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
       "      <td>{\"start_time\": \"2024-11-23T08:04:41.044939\", \"...</td>\n",
       "      <td>2024-11-23T08:04:42.304988</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'args': {'prompt': 'What is the importance o...</td>\n",
       "      <td>[{'args': {'prompt': 'What is the importance o...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1694</td>\n",
       "      <td>0.002576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Direct Query Engine</td>\n",
       "      <td>{\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...</td>\n",
       "      <td>RetrieverQueryEngine(llama_index.query_engine....</td>\n",
       "      <td>record_hash_4cbc0714b6c9b9f8febc3f6cd1a9cfaa</td>\n",
       "      <td>\"What are some good habits to develop for a su...</td>\n",
       "      <td>\"Developing good habits in areas such as eatin...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_4cbc0714b6c9b9f8feb...</td>\n",
       "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
       "      <td>{\"start_time\": \"2024-11-23T08:04:42.416744\", \"...</td>\n",
       "      <td>2024-11-23T08:04:43.389829</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'args': {'prompt': 'What are some good habit...</td>\n",
       "      <td>[{'args': {'prompt': 'What are some good habit...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1631</td>\n",
       "      <td>0.002465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Direct Query Engine</td>\n",
       "      <td>{\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...</td>\n",
       "      <td>RetrieverQueryEngine(llama_index.query_engine....</td>\n",
       "      <td>record_hash_bd2e6b060fd2e9f8f29adb947397856c</td>\n",
       "      <td>\"How can altruism be beneficial in building a ...</td>\n",
       "      <td>\"Helping others during your career journey can...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_bd2e6b060fd2e9f8f29...</td>\n",
       "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
       "      <td>{\"start_time\": \"2024-11-23T08:04:43.511277\", \"...</td>\n",
       "      <td>2024-11-23T08:04:44.267362</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'args': {'prompt': 'How can altruism be bene...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1609</td>\n",
       "      <td>0.002421</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                app_id                                           app_json  \\\n",
       "0  Direct Query Engine  {\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...   \n",
       "1  Direct Query Engine  {\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...   \n",
       "2  Direct Query Engine  {\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...   \n",
       "3  Direct Query Engine  {\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...   \n",
       "4  Direct Query Engine  {\"app_id\": \"Direct Query Engine\", \"tags\": \"-\",...   \n",
       "\n",
       "                                                type  \\\n",
       "0  RetrieverQueryEngine(llama_index.query_engine....   \n",
       "1  RetrieverQueryEngine(llama_index.query_engine....   \n",
       "2  RetrieverQueryEngine(llama_index.query_engine....   \n",
       "3  RetrieverQueryEngine(llama_index.query_engine....   \n",
       "4  RetrieverQueryEngine(llama_index.query_engine....   \n",
       "\n",
       "                                      record_id  \\\n",
       "0  record_hash_f7fff232a5841e153dd60e4e0e384fe5   \n",
       "1  record_hash_c8c4775f87a3b231bbb60b4a2e2cc0b8   \n",
       "2  record_hash_74173a7b0713c79a51d6ce6c4783522a   \n",
       "3  record_hash_4cbc0714b6c9b9f8febc3f6cd1a9cfaa   \n",
       "4  record_hash_bd2e6b060fd2e9f8f29adb947397856c   \n",
       "\n",
       "                                               input  \\\n",
       "0    \"What are the keys to building a career in AI?\"   \n",
       "1    \"How can teamwork contribute to success in AI?\"   \n",
       "2      \"What is the importance of networking in AI?\"   \n",
       "3  \"What are some good habits to develop for a su...   \n",
       "4  \"How can altruism be beneficial in building a ...   \n",
       "\n",
       "                                              output tags  \\\n",
       "0  \"Learning foundational technical skills, worki...    -   \n",
       "1  \"Teamwork can contribute to success in AI by a...    -   \n",
       "2  \"Networking is crucial in AI as it helps indiv...    -   \n",
       "3  \"Developing good habits in areas such as eatin...    -   \n",
       "4  \"Helping others during your career journey can...    -   \n",
       "\n",
       "                                         record_json  \\\n",
       "0  {\"record_id\": \"record_hash_f7fff232a5841e153dd...   \n",
       "1  {\"record_id\": \"record_hash_c8c4775f87a3b231bbb...   \n",
       "2  {\"record_id\": \"record_hash_74173a7b0713c79a51d...   \n",
       "3  {\"record_id\": \"record_hash_4cbc0714b6c9b9f8feb...   \n",
       "4  {\"record_id\": \"record_hash_bd2e6b060fd2e9f8f29...   \n",
       "\n",
       "                                           cost_json  \\\n",
       "0  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
       "1  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
       "2  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
       "3  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
       "4  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
       "\n",
       "                                           perf_json  \\\n",
       "0  {\"start_time\": \"2024-11-23T08:04:38.679417\", \"...   \n",
       "1  {\"start_time\": \"2024-11-23T08:04:39.854287\", \"...   \n",
       "2  {\"start_time\": \"2024-11-23T08:04:41.044939\", \"...   \n",
       "3  {\"start_time\": \"2024-11-23T08:04:42.416744\", \"...   \n",
       "4  {\"start_time\": \"2024-11-23T08:04:43.511277\", \"...   \n",
       "\n",
       "                           ts  Answer Relevance  Context Relevance  \\\n",
       "0  2024-11-23T08:04:39.735457               1.0               1.00   \n",
       "1  2024-11-23T08:04:40.930055               1.0               0.55   \n",
       "2  2024-11-23T08:04:42.304988               1.0               0.50   \n",
       "3  2024-11-23T08:04:43.389829               1.0               0.90   \n",
       "4  2024-11-23T08:04:44.267362               1.0                NaN   \n",
       "\n",
       "   Groundedness                             Answer Relevance_calls  \\\n",
       "0      1.000000  [{'args': {'prompt': 'What are the keys to bui...   \n",
       "1      0.266667  [{'args': {'prompt': 'How can teamwork contrib...   \n",
       "2           NaN  [{'args': {'prompt': 'What is the importance o...   \n",
       "3           NaN  [{'args': {'prompt': 'What are some good habit...   \n",
       "4           NaN  [{'args': {'prompt': 'How can altruism be bene...   \n",
       "\n",
       "                             Context Relevance_calls  \\\n",
       "0  [{'args': {'prompt': 'What are the keys to bui...   \n",
       "1  [{'args': {'prompt': 'How can teamwork contrib...   \n",
       "2  [{'args': {'prompt': 'What is the importance o...   \n",
       "3  [{'args': {'prompt': 'What are some good habit...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                  Groundedness_calls  latency  total_tokens  \\\n",
       "0  [{'args': {'source': 'PAGE 1Founder, DeepLearn...        1          2066   \n",
       "1  [{'args': {'source': 'Hopefully the previous c...        1          1676   \n",
       "2                                                NaN        1          1694   \n",
       "3                                                NaN        0          1631   \n",
       "4                                                NaN        0          1609   \n",
       "\n",
       "   total_cost  \n",
       "0    0.003123  \n",
       "1    0.002540  \n",
       "2    0.002576  \n",
       "3    0.002465  \n",
       "4    0.002421  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "64310897-179b-4081-aab8-f08a3392a078",
   "metadata": {
    "height": 47,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting dashboard ...\n",
      "Config file already exists. Skipping writing process.\n",
      "Credentials file already exists. Skipping writing process.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3259b94372640a3a7ca9b4949af96e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Accordion(children=(VBox(children=(VBox(children=(Label(value='STDOUT'), Output())), VBox(children=(Label(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dashboard started at https://s172-29-3-5p38560.lab-aws-production.deeplearning.ai/ .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Popen: returncode: None args: ['streamlit', 'run', '--server.headless=True'...>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# launches on http://localhost:8501/\n",
    "tru.run_dashboard()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4eedcef",
   "metadata": {},
   "source": [
    "## Advanced RAG pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a17ea2b",
   "metadata": {},
   "source": [
    "### 1. Sentence Window retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dae4a668-3699-4750-82f7-e53ae1bca3a7",
   "metadata": {
    "height": 64,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index.llms import OpenAI\n",
    "\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "78f7678f-358d-448d-b153-11ac8e96a7fc",
   "metadata": {
    "height": 149,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from utils import build_sentence_window_index\n",
    "\n",
    "sentence_index = build_sentence_window_index(\n",
    "    document,\n",
    "    llm,\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    save_dir=\"sentence_index\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "72f904c3-9845-4df5-9d2e-e5115160f987",
   "metadata": {
    "height": 64,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad72aea881ef4ed29b02f7f606fe407a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/799 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de88399ee79849bb92d802249e1e2dd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd9a7f0beae04ac3a47d36f001e8ab4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/443 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0373ebeba9a04ed98e46e91c1ec41acb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aadedb74d64943ee88f6d555c4c325ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63d608bc94234918ae51c75fa8ec0cea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/279 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from utils import get_sentence_window_query_engine\n",
    "\n",
    "sentence_window_engine = get_sentence_window_query_engine(sentence_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f59e2314-7cac-42f4-a552-9a8e4db641eb",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To get started on a personal project in AI, you can begin by identifying a project that aligns with your career goals and interests. It's important to choose a project that is responsible, ethical, and beneficial to people. Once you have selected a project, you can follow the steps outlined in the chapters provided, such as scoping the project, executing it with an eye toward career development, and building a portfolio that demonstrates skill progression. By following these guidelines, you can embark on a personal AI project that not only enhances your skills but also contributes positively to the field.\n"
     ]
    }
   ],
   "source": [
    "window_response = sentence_window_engine.query(\n",
    "    \"how do I get started on a personal project in AI?\"\n",
    ")\n",
    "print(str(window_response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b5c10917-8846-4e73-838d-6232c906a7db",
   "metadata": {
    "height": 115,
    "tags": []
   },
   "outputs": [],
   "source": [
    "tru.reset_database()\n",
    "\n",
    "tru_recorder_sentence_window = get_prebuilt_trulens_recorder(\n",
    "    sentence_window_engine,\n",
    "    app_id = \"Sentence Window Query Engine\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "11710e67-aba8-479e-8585-c4c611e2c1d2",
   "metadata": {
    "height": 98,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are the keys to building a career in AI?\n",
      "Learning foundational technical skills, working on projects, finding a job, and being part of a supportive community are the keys to building a career in AI.\n",
      "How can teamwork contribute to success in AI?\n",
      "Teammates play a crucial role in the success of AI projects. Working collaboratively with colleagues who are dedicated, continuously learning, and focused on using AI to benefit all people can positively influence one's own work ethic and outcomes. The ability to work effectively in a team, leveraging each member's strengths and insights, can lead to improved project outcomes and overall success in AI endeavors.\n",
      "What is the importance of networking in AI?\n",
      "Networking in AI is crucial as it can provide valuable insights, guidance, and opportunities for individuals looking to advance in the field. By connecting with professionals who have experience in AI, individuals can gain knowledge about the industry, potential career paths, and current trends. Networking also allows individuals to build relationships with others in the field, which can lead to mentorship, collaboration on projects, and even job opportunities. Overall, networking in AI can help individuals stay informed, grow their skills, and expand their professional connections.\n",
      "What are some good habits to develop for a successful career?\n",
      "Developing good habits in areas such as eating, exercise, sleep, personal relationships, work, learning, and self-care can help individuals move forward in their careers while maintaining their health.\n",
      "How can altruism be beneficial in building a career?\n",
      "Helping others and lifting them up during one's own career journey can lead to better outcomes for oneself.\n",
      "What is imposter syndrome and how does it relate to AI?\n",
      "Imposter syndrome is a phenomenon where individuals doubt their accomplishments and have a persistent fear of being exposed as a fraud. In the context of AI, newcomers to the field may experience imposter syndrome, feeling like they do not truly belong in the AI community despite their success. This can be a significant challenge for individuals in AI, as it may hinder their growth and progress in the field.\n",
      "Who are some accomplished individuals who have experienced imposter syndrome?\n",
      "Former Facebook COO Sheryl Sandberg, U.S. first lady Michelle Obama, actor Tom Hanks, and Atlassian co-CEO Mike Cannon-Brookes are some accomplished individuals who have experienced imposter syndrome.\n",
      "What is the first step to becoming good at AI?\n",
      "The first step to becoming good at AI is learning foundational technical skills.\n",
      "What are some common challenges in AI?\n",
      "Some common challenges in AI include the evolving nature of technologies, the need to continuously keep up-to-date with changing technology, the difficulty in estimating project timelines and return on investment, the iterative nature of AI projects leading to uncertainties in project management, the necessity to collaborate with stakeholders lacking expertise in AI, and the challenge of reading and understanding complex research papers in the field.\n",
      "Is it normal to find parts of AI challenging?\n",
      "It is normal to find parts of AI challenging.\n",
      "What is the right AI job for me?\n",
      "The right AI job for you can be found by following the steps outlined in Chapter 9: Finding the Right AI Job for You. It involves building a portfolio of projects that demonstrate skill progression, using a simple framework to start your AI job search, and conducting informational interviews to find the job that aligns with your career goals.\n"
     ]
    }
   ],
   "source": [
    "for question in eval_questions:\n",
    "    with tru_recorder_sentence_window as recording:\n",
    "        response = sentence_window_engine.query(question)\n",
    "        print(question)\n",
    "        print(str(response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "869d1e55-729b-45f2-a0f9-773c49d4616f",
   "metadata": {
    "height": 30,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Context Relevance</th>\n",
       "      <th>Groundedness</th>\n",
       "      <th>Answer Relevance</th>\n",
       "      <th>latency</th>\n",
       "      <th>total_cost</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>app_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Sentence Window Query Engine</th>\n",
       "      <td>0.459091</td>\n",
       "      <td>0.686255</td>\n",
       "      <td>0.990909</td>\n",
       "      <td>1.454545</td>\n",
       "      <td>0.000812</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Context Relevance  Groundedness  \\\n",
       "app_id                                                          \n",
       "Sentence Window Query Engine           0.459091      0.686255   \n",
       "\n",
       "                              Answer Relevance   latency  total_cost  \n",
       "app_id                                                                \n",
       "Sentence Window Query Engine          0.990909  1.454545    0.000812  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tru.get_leaderboard(app_ids=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0b92d0f2-2e80-48d5-92af-b3655eb03ea2",
   "metadata": {
    "height": 47,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting dashboard ...\n",
      "Config file already exists. Skipping writing process.\n",
      "Credentials file already exists. Skipping writing process.\n",
      "Dashboard already running at path: https://s172-29-3-5p38560.lab-aws-production.deeplearning.ai/\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Popen: returncode: None args: ['streamlit', 'run', '--server.headless=True'...>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# launches on http://localhost:8501/\n",
    "tru.run_dashboard()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a92e2c55",
   "metadata": {},
   "source": [
    "### 2. Auto-merging retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "558c639b-31eb-4c34-b6c4-fe6ae5717733",
   "metadata": {
    "height": 149,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from utils import build_automerging_index\n",
    "\n",
    "automerging_index = build_automerging_index(\n",
    "    documents,\n",
    "    llm,\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    save_dir=\"merging_index\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b32265f2-0247-42df-9abe-97d52f69edcf",
   "metadata": {
    "height": 98,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from utils import get_automerging_query_engine\n",
    "\n",
    "automerging_query_engine = get_automerging_query_engine(\n",
    "    automerging_index,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "918ed568-220e-4c7c-aa60-cfa58ef1fcbd",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 3ac6a45e-f783-4e7a-9e7d-48b4afc50104.\n",
      "> Parent node text: PAGE 21Building a Portfolio of \n",
      "Projects that Shows \n",
      "Skill Progression CHAPTER 6\n",
      "PROJECTS\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 9da6bdb8-3a06-4a89-b158-046a1f56d461.\n",
      "> Parent node text: PAGE 21Building a Portfolio of \n",
      "Projects that Shows \n",
      "Skill Progression CHAPTER 6\n",
      "PROJECTS\n",
      "\n",
      "Building a portfolio of AI projects involves showcasing a progression from simple to complex undertakings over time. It is important to be able to effectively communicate your thinking to demonstrate the value of your work and gain the trust of others. Identifying ideas that are worth working on is a crucial skill for an AI architect, and gaining experience by working on projects across various industries can help in building a diverse and impactful portfolio.\n"
     ]
    }
   ],
   "source": [
    "auto_merging_response = automerging_query_engine.query(\n",
    "    \"How do I build a portfolio of AI projects?\"\n",
    ")\n",
    "print(str(auto_merging_response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "db4f18a9-7b8a-4ae2-ab11-3a6a941a5afc",
   "metadata": {
    "height": 81,
    "tags": []
   },
   "outputs": [],
   "source": [
    "tru.reset_database()\n",
    "\n",
    "tru_recorder_automerging = get_prebuilt_trulens_recorder(automerging_query_engine,\n",
    "                                                         app_id=\"Automerging Query Engine\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "99cc2cfe-7096-4fa0-aa72-094bebac35a3",
   "metadata": {
    "height": 98,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function BaseQueryEngine.query at 0x7feaa620b640>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n",
      "A new object of type <class 'llama_index.retrievers.auto_merging_retriever.AutoMergingRetriever'> at 0x7fe9a4106e60 is calling an instrumented method <function BaseRetriever.retrieve at 0x7feaa620a9e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app.retriever based on other object (0x7fe9a434b070) using this function.\n",
      "A new object of type <class 'llama_index.indices.vector_store.retrievers.retriever.VectorIndexRetriever'> at 0x7fe9a427ac80 is calling an instrumented method <function BaseRetriever.retrieve at 0x7feaa620a9e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app.retriever based on other object (0x7fe9a434b070) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Merging 2 nodes into parent node.\n",
      "> Parent node id: 828f6aad-3014-4b1c-b1ce-4abc2c834f19.\n",
      "> Parent node text: PAGE 3Table of \n",
      "ContentsIntroduction: Coding AI is the New Literacy.\n",
      "Chapter 1: Three Steps to Ca...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: d578a175-96ec-468a-9237-0f59eaafb3ba.\n",
      "> Parent node text: PAGE 3Table of \n",
      "ContentsIntroduction: Coding AI is the New Literacy.\n",
      "Chapter 1: Three Steps to Ca...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function CompactAndRefine.get_response at 0x7feaa5823520>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.llm_predictor.base.LLMPredictor'> at 0x7fea44560b80 is calling an instrumented method <function LLMPredictor.predict at 0x7feab28cd120>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer.service_context.llm_predictor based on other object (0x7fea4ceb0d00) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are the keys to building a career in AI?\n",
      "Learning foundational technical skills, working on projects, finding a job, and being part of a community are essential keys to building a career in AI. Additionally, collaborating with others, influencing, and being influenced by others are critical aspects for success in AI.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How can teamwork contribute to success in AI?\n",
      "Teamwork can contribute to success in AI by allowing individuals to work together effectively on large projects. Collaboration within a team enables individuals to leverage diverse skills and perspectives, leading to more innovative solutions. Additionally, the ability to influence and be influenced by team members can enhance the quality of work and foster a more dynamic and productive work environment.\n",
      "> Merging 3 nodes into parent node.\n",
      "> Parent node id: 14ba1692-c318-4d90-871d-889be26312fe.\n",
      "> Parent node text: PAGE 35Keys to Building a Career in AI CHAPTER 10\n",
      "The path to career success in AI is more comple...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: f1c2a97f-e025-4b46-9cd2-5f948bf030f4.\n",
      "> Parent node text: PAGE 35Keys to Building a Career in AI CHAPTER 10\n",
      "The path to career success in AI is more comple...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is the importance of networking in AI?\n",
      "Networking in AI is crucial as it helps individuals build a strong professional network within the industry. This network can provide support, guidance, and opportunities for career advancement. By connecting with others in the field, individuals can gain valuable insights, access resources, and potentially open doors to new roles or projects.\n",
      "> Merging 2 nodes into parent node.\n",
      "> Parent node id: 051525b9-91f9-4a6b-8fb5-44b582d211e6.\n",
      "> Parent node text: PAGE 36Keys to Building a Career in AI CHAPTER 10\n",
      "Of all the steps in building a career, this \n",
      "on...\n",
      "\n",
      "> Merging 2 nodes into parent node.\n",
      "> Parent node id: 484ad036-ef70-46f2-b1ec-9f0829bbcb1d.\n",
      "> Parent node text: PAGE 11\n",
      "The Best Way to Build \n",
      "a New Habit\n",
      "One of my favorite books is BJ Fogg’s, Tiny Habits: Th...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 6054c2c3-7afd-455a-9843-db5e9830aec0.\n",
      "> Parent node text: PAGE 36Keys to Building a Career in AI CHAPTER 10\n",
      "Of all the steps in building a career, this \n",
      "on...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 21878a28-31de-46c8-8e4e-e56500eec30a.\n",
      "> Parent node text: PAGE 11\n",
      "The Best Way to Build \n",
      "a New Habit\n",
      "One of my favorite books is BJ Fogg’s, Tiny Habits: Th...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are some good habits to develop for a successful career?\n",
      "Good habits to develop for a successful career include habits related to eating well, exercising regularly, getting enough sleep, maintaining positive personal relationships, consistently working towards learning and self-improvement, and practicing self-care. These habits can help individuals progress in their careers while also ensuring their overall well-being.\n",
      "> Merging 2 nodes into parent node.\n",
      "> Parent node id: 475135c3-7f00-41fb-9e40-aa619e524de7.\n",
      "> Parent node text: PAGE 30Finding someone to interview isn’t always easy, but many people who are in senior position...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: afc65b40-fd2b-4166-a85c-9f4ca7529525.\n",
      "> Parent node text: PAGE 30Finding someone to interview isn’t always easy, but many people who are in senior position...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How can altruism be beneficial in building a career?\n",
      "Altruism can be beneficial in building a career by creating a positive impact on others, fostering strong relationships within professional networks, and potentially leading to opportunities for personal growth and advancement.\n",
      "> Merging 5 nodes into parent node.\n",
      "> Parent node id: d03b4df3-7f3d-41a0-97be-9a5f0530b162.\n",
      "> Parent node text: PAGE 38Before we dive into the final chapter of this book, I’d like to address the serious matter...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 03097e6e-aab5-4390-8f45-8605fdc846f0.\n",
      "> Parent node text: PAGE 37Overcoming Imposter \n",
      "SyndromeCHAPTER 11\n",
      "\n",
      "> Merging 3 nodes into parent node.\n",
      "> Parent node id: fdb5021e-7d3d-47c5-a969-275fa71dd0d7.\n",
      "> Parent node text: PAGE 39My three-year-old daughter (who can barely count to 12) regularly tries to teach things to...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: b47bfdd1-6dff-40ac-809a-88eba445105a.\n",
      "> Parent node text: PAGE 38Before we dive into the final chapter of this book, I’d like to address the serious matter...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 0ef57b6b-e869-4506-a4e4-8b96641e59f7.\n",
      "> Parent node text: PAGE 37Overcoming Imposter \n",
      "SyndromeCHAPTER 11\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: a0610202-aff3-405f-a9a1-475e64afd1b7.\n",
      "> Parent node text: PAGE 39My three-year-old daughter (who can barely count to 12) regularly tries to teach things to...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is imposter syndrome and how does it relate to AI?\n",
      "Imposter syndrome is when individuals doubt their accomplishments and have a persistent fear of being exposed as a fraud, despite evidence of their success. In the context of AI, newcomers to the field may experience imposter syndrome as they navigate the technically complex nature of artificial intelligence. It is common for individuals in the AI community, including accomplished professionals, to feel like they do not belong or question their abilities. The message conveyed is that experiencing challenges and setbacks in AI is normal, and it should not deter anyone from pursuing a career or involvement in the AI community.\n",
      "> Merging 3 nodes into parent node.\n",
      "> Parent node id: d03b4df3-7f3d-41a0-97be-9a5f0530b162.\n",
      "> Parent node text: PAGE 38Before we dive into the final chapter of this book, I’d like to address the serious matter...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 03097e6e-aab5-4390-8f45-8605fdc846f0.\n",
      "> Parent node text: PAGE 37Overcoming Imposter \n",
      "SyndromeCHAPTER 11\n",
      "\n",
      "> Merging 3 nodes into parent node.\n",
      "> Parent node id: fdb5021e-7d3d-47c5-a969-275fa71dd0d7.\n",
      "> Parent node text: PAGE 39My three-year-old daughter (who can barely count to 12) regularly tries to teach things to...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: b47bfdd1-6dff-40ac-809a-88eba445105a.\n",
      "> Parent node text: PAGE 38Before we dive into the final chapter of this book, I’d like to address the serious matter...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 0ef57b6b-e869-4506-a4e4-8b96641e59f7.\n",
      "> Parent node text: PAGE 37Overcoming Imposter \n",
      "SyndromeCHAPTER 11\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: a0610202-aff3-405f-a9a1-475e64afd1b7.\n",
      "> Parent node text: PAGE 39My three-year-old daughter (who can barely count to 12) regularly tries to teach things to...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Who are some accomplished individuals who have experienced imposter syndrome?\n",
      "Sheryl Sandberg, Michelle Obama, Tom Hanks, and Mike Cannon-Brookes are some accomplished individuals who have experienced imposter syndrome.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is the first step to becoming good at AI?\n",
      "The first step to becoming good at AI is to suck at it.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are some common challenges in AI?\n",
      "Common challenges in AI include the highly iterative nature of AI projects, which can make it difficult to plan for building a system when the time needed to achieve the target accuracy is uncertain. Additionally, technical challenges in AI can be daunting, leading to feelings of imposter syndrome among those in the field.\n",
      "> Merging 3 nodes into parent node.\n",
      "> Parent node id: d03b4df3-7f3d-41a0-97be-9a5f0530b162.\n",
      "> Parent node text: PAGE 38Before we dive into the final chapter of this book, I’d like to address the serious matter...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: b47bfdd1-6dff-40ac-809a-88eba445105a.\n",
      "> Parent node text: PAGE 38Before we dive into the final chapter of this book, I’d like to address the serious matter...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n",
      "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x7fe93ac54d00 is calling an instrumented method <function RetrieverQueryEngine.retrieve at 0x7feaa126ea70>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app based on other object (0x7fe9a4386e00) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is it normal to find parts of AI challenging?\n",
      "It is normal to find parts of AI challenging.\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 507c7827-02db-44b8-9c53-0d3246d02f5c.\n",
      "> Parent node text: PAGE 31Finding the Right \n",
      "AI Job for YouCHAPTER 9\n",
      "JOBS\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 9843b1a7-09a9-4fdb-839f-b69fd74692a6.\n",
      "> Parent node text: If you’re leaving \n",
      "a job, exit gracefully. Give your employer ample notice, give your full effort...\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 6eae300e-19c5-4d12-8087-b07a95d54d59.\n",
      "> Parent node text: PAGE 28Using Informational \n",
      "Interviews to Find \n",
      "the Right JobCHAPTER 8\n",
      "JOBS\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 5cb5a2e0-faa7-49ce-bd2f-da1d10556b7e.\n",
      "> Parent node text: PAGE 31Finding the Right \n",
      "AI Job for YouCHAPTER 9\n",
      "JOBS\n",
      "\n",
      "> Merging 1 nodes into parent node.\n",
      "> Parent node id: 9f62d133-9815-4388-8b1f-98a935d0ef6e.\n",
      "> Parent node text: PAGE 28Using Informational \n",
      "Interviews to Find \n",
      "the Right JobCHAPTER 8\n",
      "JOBS\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x7fe93ac54c70 is calling an instrumented method <function Refine.get_response at 0x7feaa4c3f2e0>. The path of this call may be incorrect.\n",
      "Guessing path of new object is app._response_synthesizer based on other object (0x7fe9a4387f10) using this function.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is the right AI job for me?\n",
      "The right AI job for you would likely depend on whether you are looking to switch roles, industries, or both. If you are seeking your first job in AI, it may be easier to transition by switching either roles or industries rather than attempting both simultaneously.\n"
     ]
    }
   ],
   "source": [
    "for question in eval_questions:\n",
    "    with tru_recorder_automerging as recording:\n",
    "        response = automerging_query_engine.query(question)\n",
    "        print(question)\n",
    "        print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5404dec1-60ca-42fa-ac13-793a5423aa64",
   "metadata": {
    "height": 30,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Context Relevance</th>\n",
       "      <th>Groundedness</th>\n",
       "      <th>Answer Relevance</th>\n",
       "      <th>latency</th>\n",
       "      <th>total_cost</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>app_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Automerging Query Engine</th>\n",
       "      <td>0.685</td>\n",
       "      <td>0.73</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Context Relevance  Groundedness  Answer Relevance  \\\n",
       "app_id                                                                        \n",
       "Automerging Query Engine              0.685          0.73               1.0   \n",
       "\n",
       "                          latency  total_cost  \n",
       "app_id                                         \n",
       "Automerging Query Engine      2.0    0.000863  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tru.get_leaderboard(app_ids=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0f545d41-0d98-446f-8214-8b59bef08d6c",
   "metadata": {
    "height": 47,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting dashboard ...\n",
      "Config file already exists. Skipping writing process.\n",
      "Credentials file already exists. Skipping writing process.\n",
      "Dashboard already running at path: https://s172-29-3-5p38560.lab-aws-production.deeplearning.ai/\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Popen: returncode: None args: ['streamlit', 'run', '--server.headless=True'...>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# launches on http://localhost:8501/\n",
    "tru.run_dashboard()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
